import os
from datetime import datetime
from json.decoder import JSONDecodeError

import logzero
import requests
from logzero import logger
from tqdm import tqdm

from config import COLLECTION_FEED
from utils import get_mongo_collection, record_run_status

pwd = os.path.dirname(os.path.abspath(__file__))
current_module = os.path.splitext(os.path.basename(__file__))[0]
COLLECTION_NAME = current_module
logzero.logfile(f"/tmp/{current_module}.log", maxBytes=30_000_000, backupCount=3, encoding='utf-8')

db = get_mongo_collection(COLLECTION_NAME)

ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 ' \
     'Safari/537.36'
headers = {'User-Agent': ua}

insert_count = 0


def spider(feeds):
    for user_name, name in tqdm(feeds):
        p = 0
        stop = False
        while not stop:
            url = f'https://zhuanlan.zhihu.com/api/columns/{user_name}/posts?limit=20&offset={p}'
            logger.info(f'访问 {url}')
            try:
                posts = requests.get(url, headers=headers).json()
            except JSONDecodeError:
                logger.error(f'{user_name} 404了')
                break
            if len(posts) == 0:
                logger.info('翻页完毕')
                break
            for data in posts:
                if data.get('titleImage') == '':
                    continue
                data.update({'user_name': user_name, 'name': name, 'update_time': datetime.now()})
                stop = store_data(data)
                if stop:
                    break
            p += 20
            logger.info('翻页')
    logger.info('完成')


def get_feeds():
    db_feed = get_mongo_collection(COLLECTION_FEED)
    cursor = db_feed.find({'purpose': '知乎专栏'})
    feeds = []
    for cu in cursor:
        feeds.append((cu['user_name'], cu['name']))
    return feeds


def store_data(data):
    global insert_count
    stop = False
    href = data['href']
    old_record = db.find_one({'href': href})
    if old_record:
        logger.info(f'{href} 已经存在，不再爬取')
        stop = True
    else:
        logger.info(f'{href} 不存在，现在插入')
        logger.info(data)
        db.insert_one(data)
        insert_count += 1
    return stop


@record_run_status(current_module)
def main():
    feeds = get_feeds()
    spider(feeds)
    return insert_count


if __name__ == '__main__':
    main()
